package com.bblocks.common.data.support;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.bblocks.area.AreaVo;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * <p>Project: bblocks-pro - AreaSpiderWithCache</p>
 * <p>描述：爬取省市区街道社区（5级）-本地缓存以优化提取速度（二次提取速度）</p>
 *
 * @Author Zhao [125043150@qq.com]
 * @Date 2024/3/18 17:05
 * @Version 1.0
 * @since 8
 */
@Slf4j
public class AreaSpiderWithCache extends AreaSpider{
    private static final Cache<String, Object> cache = Caffeine.newBuilder()
            .expireAfterWrite(60, TimeUnit.MINUTES)
            .maximumSize(10_000)
            .build();

    private List<AreaVo> result;

    /**
     *
     * @param year
     * @param containLv
     * @param list
     */
    public AreaSpiderWithCache(int year, int containLv, List<AreaVo> list) {
        super(year, containLv);

        this.result = list;
    }

    protected boolean addFollowsUrl(Page page, AreaVo parent, int targetLv) {
        String newUrl = parent.getChildUrl();
        if(StrUtil.isEmpty(newUrl))
            return false;
        List<AreaVo> areas = (List<AreaVo>)cache.getIfPresent(newUrl);
        if(CollUtil.isEmpty(areas)){
            page.addTargetRequest(newUrl);
            log.info("即将采集地址:{}" , newUrl);
            return true;
        }else {
            /**
             * 子集 添加采集
             */
            if(targetLv < getContainLv()){
                areas.forEach(a->{
                    if(StrUtil.isNotEmpty(a.getChildUrl())){
                        addFollowsUrl(page,a,targetLv + 1);
                    }
                });
            }
            this.result.addAll(areas);
            return false;
        }
    }



    public static void main(String[] args) {
        long b = System.currentTimeMillis();
        List<AreaVo> result = Lists.newArrayList();
        AreaSpiderWithCache.LogPipeline logPipeline = new AreaSpiderWithCache.LogPipeline(result);
        Spider.create(new AreaSpiderWithCache(2022, 1,result))
                .addUrl(SPIDER_URL + "index.html")
                .addPipeline(logPipeline)
                .thread(2)
                .run();

        log.info("采集到1:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));

        b = System.currentTimeMillis();
        result.clear();
        logPipeline = new AreaSpiderWithCache.LogPipeline(result);
        Spider.create(new AreaSpiderWithCache(2022, 1,result))
                .addUrl(SPIDER_URL + "index.html")
                .addPipeline(logPipeline)
                .thread(2)
                .run();
        log.info("采集到1:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));

    }

    @Data
    public static class LogPipeline implements Pipeline {
        List<AreaVo> result;
        /**
         * 失败次数 0=成功，
         */
        int failNum  ;
        public LogPipeline(List<AreaVo> result ){
            this.result = result;
            this.failNum = 0;
        }

        @Override
        public void process(ResultItems resultItems, Task task) {
            String url = resultItems.getRequest().getUrl();
            List<AreaVo> areas = resultItems.get("area");
            log.info("地区总数：{}" , areas.size());


            if (areas.isEmpty()) {//url采集失败
                log.info(url + " 此页面未爬取数据,请稍后重试!");
                failNum ++;
            } else {
                areas.stream().forEach(a -> {
                    log.info("{}" , a);
                });
                //缓存已采集数据
                cache.put(url,areas);

                this.result.addAll(areas);
            }
        }
    }
}
